{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow\n",
    "import tensorflow.compat.v1 as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "![](picture/CNN.png)\n",
    "![](picture/CNN2.png)\n",
    "![](picture/离散卷积.png)\n",
    "![](picture/卷积.png)\n",
    "![](picture/卷积2.png)\n",
    "![](picture/卷积3.png)\n",
    "![](picture/卷积四.png)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "def convld(x,w,p,s=1):\n",
    "    w_rot=np.array(w[::-1])\n",
    "    x_padded=np.array(x)\n",
    "    if p>0:\n",
    "        zero_pad=np.zeros(shape=p)\n",
    "        x_padded=np.concatenate([zero_pad,x_padded,zero_pad])\n",
    "    res=[]\n",
    "    for i in range(0,int(len(x)/s),s):\n",
    "        res.append(np.sum(x_padded[i:i+w_rot.shape[0]]*w_rot))\n",
    "    return np.array(res)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 5. 14. 16. 26. 24. 34. 19. 22.]\n",
      "[ 5 14 16 26 24 34 19 22]\n"
     ]
    }
   ],
   "source": [
    "x=[1,3,2,4,5,6,1,3]\n",
    "w=[1,0,3,1,2]\n",
    "print(convld(x,w,p=2,s=1))\n",
    "print(np.convolve(x,w,mode='same'))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "![](picture/卷积5.png)\n",
    "![](picture/卷积6.png)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "def conv2d(x,w,p=(0,0),s=(1,1)):\n",
    "    w_rot=np.array(w)[::-1,::-1]\n",
    "    x_orig=np.array(x)\n",
    "    n1=x_orig.shape[0]+2*p[0]\n",
    "    n2=x_orig.shape[1]+2*p[1]\n",
    "    x_padded=np.zeros(shape=(n1,n2))\n",
    "    x_padded[p[0]:p[0]+x_orig.shape[0],p[1]:p[1]+x_orig.shape[1]]=x_orig\n",
    "    res=[]\n",
    "    for i in range(0,int((x_padded.shape[0]-w_rot.shape[0])/s[0]+1),s[0]):\n",
    "        res.append([])\n",
    "        for j in range(0,int((x_padded.shape[1]-w_rot.shape[1])/s[1]+1),s[1]):\n",
    "            x_sub=x_padded[i:i+w_rot.shape[0],j:j+w_rot.shape[1]]\n",
    "            res[-1].append(np.sum(x_sub*w_rot))\n",
    "    return (np.array(res))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[10. 24. 33. 14.]\n",
      " [23. 24. 37. 14.]\n",
      " [17. 36. 30. 19.]\n",
      " [11. 21. 18. 10.]]\n"
     ]
    }
   ],
   "source": [
    "x=[[1,2,3,4],[5,6,1,3],[1,6,0,3],[3,4,3,2]]\n",
    "w=[[1,0,3],[1,2,1],[0,1,1]]\n",
    "print(conv2d(x,w,p=(1,1,),s=(1,1)))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[10 24 33 14]\n",
      " [23 24 37 14]\n",
      " [17 36 30 19]\n",
      " [11 21 18 10]]\n"
     ]
    }
   ],
   "source": [
    "import scipy.signal\n",
    "print(scipy.signal.convolve2d(x,w,mode='same'))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "![](picture/子采样.png)\n",
    "![](picture/池化.png)\n",
    "![](picture/卷积神经网络.png)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [],
   "source": [
    "#处理多个输入维度的数据\n",
    "from skimage import io\n",
    "import matplotlib.pyplot as plt\n",
    "img=io.imread('data/example-image.png')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uint8\n",
      "(252, 221, 4)\n",
      "[[[179 134 110]\n",
      "  [182 136 112]]\n",
      "\n",
      " [[180 135 111]\n",
      "  [182 137 113]]]\n"
     ]
    }
   ],
   "source": [
    "print(img.dtype)\n",
    "print(img.shape)\n",
    "print(img[100:102,100:102,0:3])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "![](picture/卷积7.png)\n",
    "![](picture/卷积8.png)\n",
    "![](picture/正则化淘汰神经网络.png)\n",
    "![](picture/正则化淘汰神经网络2.png)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "![](picture/卷积9.png)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "mnist=np.load(\"data/mnist_scaled.npz\")#标准化后的数据\n",
    "X_train,y_train,X_test,y_test=[mnist[s] for s in mnist]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "X_train,y_train,X_valid,y_valid=X_train[:50000,:],y_train[:50000],X_train[50000:,:],y_train[50000:]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 784)\n",
      "(50000,)\n",
      "(10000, 784)\n",
      "(10000,)\n",
      "(10000, 784)\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(X_valid.shape)\n",
    "print(y_valid.shape)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "#定义一个产生小批量数据的函数\n",
    "\n",
    "def batch_generator(X,y,batch_size=64,shuffle=False,random_seed=None):\n",
    "    idx=np.arange(y.shape[0])\n",
    "    if shuffle:\n",
    "        rng=np.random.RandomState(random_seed)\n",
    "        rng.shuffle(idx)\n",
    "        X=X[idx]\n",
    "        y=y[idx]\n",
    "    for i in range(0,X.shape[0],batch_size):\n",
    "        yield X[i:i+batch_size,:],y[i:i+batch_size]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "outputs": [],
   "source": [
    "#使用底层API实现CNN\n",
    "def conv_layer(input_tensor,name,kernel_size,n_output_channels,padding_mode=\"SAME\",strides=(1,1,1,1)):\n",
    "    with tf.variable_scope(name):\n",
    "        ## get n_channels\n",
    "\n",
    "        input_shape=input_tensor.get_shape().as_list()\n",
    "        n_input_channels=input_shape[-1]\n",
    "        weights_shape=list(kernel_size)+[n_input_channels,n_output_channels]\n",
    "        weights=tf.get_variable(name='weights',shape=weights_shape)\n",
    "        print(weights)\n",
    "        biases=tf.get_variable(name='biases',initializer=tf.zeros(shape=[n_output_channels]))\n",
    "        print(biases)\n",
    "        conv=tf.nn.conv2d(input=input_tensor,\n",
    "                          filter=weights,\n",
    "                          strides=strides,\n",
    "                          padding=padding_mode)\n",
    "        print(conv)\n",
    "        conv=tf.nn.bias_add(conv,biases,name='net_pre-activation')\n",
    "        print(conv)\n",
    "        conv=tf.nn.relu(conv,name='activation')\n",
    "        print(conv)\n",
    "        return conv"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.Variable 'convtest/weights:0' shape=(3, 3, 1, 32) dtype=float32>\n",
      "<tf.Variable 'convtest/biases:0' shape=(32,) dtype=float32>\n",
      "Tensor(\"convtest/Conv2D:0\", shape=(None, 28, 28, 32), dtype=float32)\n",
      "Tensor(\"convtest/net_pre-activation:0\", shape=(None, 28, 28, 32), dtype=float32)\n",
      "Tensor(\"convtest/activation:0\", shape=(None, 28, 28, 32), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "g=tf.Graph()\n",
    "with g.as_default():\n",
    "    x=tf.placeholder(tf.float32,shape=[None,28,28,1])\n",
    "    conv_layer(x,name='convtest',\n",
    "               kernel_size=(3,3),\n",
    "               n_output_channels=32)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [],
   "source": [
    "del g,x"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [],
   "source": [
    "#定义全连接层\n",
    "def fc_layer(input_tensor,name,\n",
    "             n_output_units,activation_fn=None):\n",
    "    with tf.variable_scope(name):\n",
    "        input_shape=input_tensor.get_shape().as_list()[1:]\n",
    "        n_input_units=np.prod(input_shape)\n",
    "        if len(input_shape)>1:\n",
    "            input_tensor=tf.reshape(input_tensor,shape=(-1,n_input_units))\n",
    "        weights_shape=[n_input_units,n_output_units]\n",
    "        weights=tf.get_variable(name='weights',shape=weights_shape)\n",
    "        print(weights)\n",
    "        biases=tf.get_variable(name='biases',initializer=tf.zeros(shape=[n_output_units]))\n",
    "        print(biases)\n",
    "        layer=tf.matmul(input_tensor,weights)\n",
    "        print(layer)\n",
    "        layer=tf.nn.bias_add(layer,biases,name='net_pre-activation')\n",
    "        print(layer)\n",
    "        if activation_fn==None:\n",
    "            return layer\n",
    "        layer=activation_fn(layer,name='activation')\n",
    "        print(layer)\n",
    "        return layer"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.Variable 'fctest/weights:0' shape=(784, 32) dtype=float32>\n",
      "<tf.Variable 'fctest/biases:0' shape=(32,) dtype=float32>\n",
      "Tensor(\"fctest/MatMul:0\", shape=(None, 32), dtype=float32)\n",
      "Tensor(\"fctest/net_pre-activation:0\", shape=(None, 32), dtype=float32)\n",
      "Tensor(\"fctest/activation:0\", shape=(None, 32), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "g=tf.Graph()\n",
    "with g.as_default():\n",
    "    x=tf.placeholder(tf.float32,\n",
    "                     name='x',shape=[None,28,28,1])\n",
    "    fc_layer(x,name='fctest',n_output_units=32,\n",
    "             activation_fn=tf.nn.relu)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [],
   "source": [
    "del g,x"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "outputs": [],
   "source": [
    "def build_cnn(learning_rate=1e-4):\n",
    "    tf_x=tf.placeholder(tf.float32,shape=[None,784],\n",
    "                        name='tf_x')\n",
    "    tf_y=tf.placeholder(tf.int32,shape=[None],\n",
    "                        name='tf_y')\n",
    "    tf_x_image=tf.reshape(tf_x,shape=[-1,28,28,1],name='tf_x_reshaped')\n",
    "    tf_y_onehot=tf.one_hot(indices=tf_y,depth=10,dtype=tf.float32,\n",
    "                           name='tf_y_onehot')\n",
    "    print('\\nBuilding 1st layer')\n",
    "    h1=conv_layer(tf_x_image,name='conv_1',\n",
    "                  kernel_size=(5,5),\n",
    "                  padding_mode='VALID',\n",
    "                  n_output_channels=32)\n",
    "    ##maxpooling\n",
    "    h1_pool=tf.nn.max_pool(h1,\n",
    "                           ksize=[1,2,2,1],\n",
    "                           strides=[1,2,2,1],\n",
    "                           padding='SAME')\n",
    "    ##第二层卷积层\n",
    "    print('\\nBuilding 2nd layer:')\n",
    "    h2=conv_layer(h1_pool,name='conv2',\n",
    "                  kernel_size=(5,5),\n",
    "                  padding_mode=\"VALID\",\n",
    "                  n_output_channels=64)\n",
    "    h2_pool=tf.nn.max_pool(h2,ksize=[1,2,2,1],\n",
    "                           strides=[1,2,2,1],\n",
    "                           padding=\"SAME\")\n",
    "    ##第三层全连接层\n",
    "    print('\\nBuilding 3rd layer:')\n",
    "    h3=fc_layer(h2_pool,name='fc_3',\n",
    "                n_output_units=1024,\n",
    "                activation_fn=tf.nn.relu)\n",
    "    ##Drop out\n",
    "    keep_prob=tf.placeholder(tf.float32,name='fc_keep_prob')\n",
    "    h3_drop=tf.nn.dropout(h3,keep_prob=keep_prob,\n",
    "                          name='dropout_layer')\n",
    "    ##全连接层\n",
    "    print('\\nBuilding 4th layer:')\n",
    "    h4=fc_layer(h3_drop,name='fc_4',\n",
    "                n_output_units=10,\n",
    "                activation_fn=None)\n",
    "    ##Predictions\n",
    "    predictions={'probabilities':tf.nn.softmax(h4,name='probabilities'),\n",
    "                 'labels':tf.cast(tf.argmax(h4,axis=1),tf.int32,\n",
    "                                  name='labels')}\n",
    "    cross_entropy_loss=tf.reduce_mean(\n",
    "        tf.nn.softmax_cross_entropy_with_logits(logits=h4,labels=tf_y_onehot),\n",
    "        name='cross_entropy_loss'\n",
    "    )\n",
    "    optimizer=tf.train.AdamOptimizer(learning_rate)\n",
    "    optimizer=optimizer.minimize(cross_entropy_loss,name='train_op')\n",
    "    correct_predictions=tf.equal(predictions['labels'],\n",
    "                                 tf_y,name='correct_preds')\n",
    "    accuracy=tf.reduce_mean(\n",
    "        tf.cast(correct_predictions,tf.float32),\n",
    "        name='accuracy'\n",
    "    )"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "outputs": [],
   "source": [
    "def save(saver, sess, epoch, path='./model/'):\n",
    "    if not os.path.isdir(path):\n",
    "        os.makedirs(path)\n",
    "    print('Saving model in %s' % path)\n",
    "    saver.save(sess, os.path.join(path,'cnn-model.ckpt'),\n",
    "               global_step=epoch)\n",
    "\n",
    "\n",
    "def load(saver, sess, path, epoch):\n",
    "    print('Loading model from %s' % path)\n",
    "    saver.restore(sess, os.path.join(\n",
    "            path, 'cnn-model.ckpt-%d' % epoch))\n",
    "\n",
    "\n",
    "def train(sess, training_set, validation_set=None,\n",
    "          initialize=True, epochs=20, shuffle=True,\n",
    "          dropout=0.5, random_seed=None):\n",
    "\n",
    "    X_data = np.array(training_set[0])\n",
    "    y_data = np.array(training_set[1])\n",
    "    training_loss = []\n",
    "\n",
    "    ## initialize variables\n",
    "    if initialize:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    np.random.seed(random_seed) # for shuflling in batch_generator\n",
    "    for epoch in range(1, epochs+1):\n",
    "        batch_gen = batch_generator(\n",
    "                        X_data, y_data,\n",
    "                        shuffle=shuffle)\n",
    "        avg_loss = 0.0\n",
    "        for i,(batch_x,batch_y) in enumerate(batch_gen):\n",
    "            feed = {'tf_x:0': batch_x,\n",
    "                    'tf_y:0': batch_y,\n",
    "                    'fc_keep_prob:0': dropout}\n",
    "            loss, _ = sess.run(\n",
    "                    ['cross_entropy_loss:0', 'train_op'],\n",
    "                    feed_dict=feed)\n",
    "            avg_loss += loss\n",
    "\n",
    "        training_loss.append(avg_loss / (i+1))\n",
    "        print('Epoch %02d Training Avg. Loss: %7.3f' % (\n",
    "            epoch, avg_loss), end=' ')\n",
    "        if validation_set is not None:\n",
    "            feed = {'tf_x:0': validation_set[0],\n",
    "                    'tf_y:0': validation_set[1],\n",
    "                    'fc_keep_prob:0':1.0}\n",
    "            valid_acc = sess.run('accuracy:0', feed_dict=feed)\n",
    "            print(' Validation Acc: %7.3f' % valid_acc)\n",
    "        else:\n",
    "            print()\n",
    "\n",
    "\n",
    "def predict(sess, X_test, return_proba=False):\n",
    "    feed = {'tf_x:0': X_test,\n",
    "            'fc_keep_prob:0': 1.0}\n",
    "    if return_proba:\n",
    "        return sess.run('probabilities:0', feed_dict=feed)\n",
    "    else:\n",
    "        return sess.run('labels:0', feed_dict=feed)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Building 1st layer\n",
      "<tf.Variable 'conv_1/weights:0' shape=(5, 5, 1, 32) dtype=float32>\n",
      "<tf.Variable 'conv_1/biases:0' shape=(32,) dtype=float32>\n",
      "Tensor(\"conv_1/Conv2D:0\", shape=(None, 24, 24, 32), dtype=float32)\n",
      "Tensor(\"conv_1/net_pre-activation:0\", shape=(None, 24, 24, 32), dtype=float32)\n",
      "Tensor(\"conv_1/activation:0\", shape=(None, 24, 24, 32), dtype=float32)\n",
      "\n",
      "Building 2nd layer:\n",
      "<tf.Variable 'conv2/weights:0' shape=(5, 5, 32, 64) dtype=float32>\n",
      "<tf.Variable 'conv2/biases:0' shape=(64,) dtype=float32>\n",
      "Tensor(\"conv2/Conv2D:0\", shape=(None, 8, 8, 64), dtype=float32)\n",
      "Tensor(\"conv2/net_pre-activation:0\", shape=(None, 8, 8, 64), dtype=float32)\n",
      "Tensor(\"conv2/activation:0\", shape=(None, 8, 8, 64), dtype=float32)\n",
      "\n",
      "Building 3rd layer:\n",
      "<tf.Variable 'fc_3/weights:0' shape=(1024, 1024) dtype=float32>\n",
      "<tf.Variable 'fc_3/biases:0' shape=(1024,) dtype=float32>\n",
      "Tensor(\"fc_3/MatMul:0\", shape=(None, 1024), dtype=float32)\n",
      "Tensor(\"fc_3/net_pre-activation:0\", shape=(None, 1024), dtype=float32)\n",
      "Tensor(\"fc_3/activation:0\", shape=(None, 1024), dtype=float32)\n",
      "\n",
      "Building 4th layer:\n",
      "<tf.Variable 'fc_4/weights:0' shape=(1024, 10) dtype=float32>\n",
      "<tf.Variable 'fc_4/biases:0' shape=(10,) dtype=float32>\n",
      "Tensor(\"fc_4/MatMul:0\", shape=(None, 10), dtype=float32)\n",
      "Tensor(\"fc_4/net_pre-activation:0\", shape=(None, 10), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "## create a graph\n",
    "random_seed = 123\n",
    "\n",
    "np.random.seed(random_seed)\n",
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "    tf.set_random_seed(random_seed)\n",
    "    ## build the graph\n",
    "    build_cnn()\n",
    "\n",
    "    ## saver:\n",
    "    saver = tf.train.Saver()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 01 Training Avg. Loss: 307.847  Validation Acc:   0.972\n",
      "Epoch 02 Training Avg. Loss:  79.140  Validation Acc:   0.982\n",
      "Epoch 03 Training Avg. Loss:  53.798  Validation Acc:   0.985\n",
      "Epoch 04 Training Avg. Loss:  41.890  Validation Acc:   0.987\n",
      "Epoch 05 Training Avg. Loss:  34.263  Validation Acc:   0.989\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Input \u001B[0;32mIn [56]\u001B[0m, in \u001B[0;36m<cell line: 1>\u001B[0;34m()\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m tf\u001B[38;5;241m.\u001B[39mSession(graph\u001B[38;5;241m=\u001B[39mg) \u001B[38;5;28;01mas\u001B[39;00m sess:\n\u001B[0;32m----> 2\u001B[0m     \u001B[43mtrain\u001B[49m\u001B[43m(\u001B[49m\u001B[43msess\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m      3\u001B[0m \u001B[43m          \u001B[49m\u001B[43mtraining_set\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mX_train\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43my_train\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m      4\u001B[0m \u001B[43m          \u001B[49m\u001B[43mvalidation_set\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mX_valid\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43my_valid\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m      5\u001B[0m \u001B[43m          \u001B[49m\u001B[43minitialize\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m      6\u001B[0m \u001B[43m          \u001B[49m\u001B[43mrandom_seed\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;241;43m123\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[1;32m      7\u001B[0m     save(saver, sess, epoch\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m20\u001B[39m)\n",
      "Input \u001B[0;32mIn [45]\u001B[0m, in \u001B[0;36mtrain\u001B[0;34m(sess, training_set, validation_set, initialize, epochs, shuffle, dropout, random_seed)\u001B[0m\n\u001B[1;32m     33\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m i,(batch_x,batch_y) \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28menumerate\u001B[39m(batch_gen):\n\u001B[1;32m     34\u001B[0m     feed \u001B[38;5;241m=\u001B[39m {\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mtf_x:0\u001B[39m\u001B[38;5;124m'\u001B[39m: batch_x,\n\u001B[1;32m     35\u001B[0m             \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mtf_y:0\u001B[39m\u001B[38;5;124m'\u001B[39m: batch_y,\n\u001B[1;32m     36\u001B[0m             \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mfc_keep_prob:0\u001B[39m\u001B[38;5;124m'\u001B[39m: dropout}\n\u001B[0;32m---> 37\u001B[0m     loss, _ \u001B[38;5;241m=\u001B[39m \u001B[43msess\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrun\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m     38\u001B[0m \u001B[43m            \u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mcross_entropy_loss:0\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mtrain_op\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     39\u001B[0m \u001B[43m            \u001B[49m\u001B[43mfeed_dict\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfeed\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     40\u001B[0m     avg_loss \u001B[38;5;241m+\u001B[39m\u001B[38;5;241m=\u001B[39m loss\n\u001B[1;32m     42\u001B[0m training_loss\u001B[38;5;241m.\u001B[39mappend(avg_loss \u001B[38;5;241m/\u001B[39m (i\u001B[38;5;241m+\u001B[39m\u001B[38;5;241m1\u001B[39m))\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:967\u001B[0m, in \u001B[0;36mBaseSession.run\u001B[0;34m(self, fetches, feed_dict, options, run_metadata)\u001B[0m\n\u001B[1;32m    964\u001B[0m run_metadata_ptr \u001B[38;5;241m=\u001B[39m tf_session\u001B[38;5;241m.\u001B[39mTF_NewBuffer() \u001B[38;5;28;01mif\u001B[39;00m run_metadata \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[1;32m    966\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 967\u001B[0m   result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_run\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfetches\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfeed_dict\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43moptions_ptr\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    968\u001B[0m \u001B[43m                     \u001B[49m\u001B[43mrun_metadata_ptr\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    969\u001B[0m   \u001B[38;5;28;01mif\u001B[39;00m run_metadata:\n\u001B[1;32m    970\u001B[0m     proto_data \u001B[38;5;241m=\u001B[39m tf_session\u001B[38;5;241m.\u001B[39mTF_GetBuffer(run_metadata_ptr)\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:1190\u001B[0m, in \u001B[0;36mBaseSession._run\u001B[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001B[0m\n\u001B[1;32m   1187\u001B[0m \u001B[38;5;66;03m# We only want to really perform the run if fetches or targets are provided,\u001B[39;00m\n\u001B[1;32m   1188\u001B[0m \u001B[38;5;66;03m# or if the call is a partial run that specifies feeds.\u001B[39;00m\n\u001B[1;32m   1189\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m final_fetches \u001B[38;5;129;01mor\u001B[39;00m final_targets \u001B[38;5;129;01mor\u001B[39;00m (handle \u001B[38;5;129;01mand\u001B[39;00m feed_dict_tensor):\n\u001B[0;32m-> 1190\u001B[0m   results \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_do_run\u001B[49m\u001B[43m(\u001B[49m\u001B[43mhandle\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfinal_targets\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfinal_fetches\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1191\u001B[0m \u001B[43m                         \u001B[49m\u001B[43mfeed_dict_tensor\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43moptions\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrun_metadata\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1192\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m   1193\u001B[0m   results \u001B[38;5;241m=\u001B[39m []\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:1370\u001B[0m, in \u001B[0;36mBaseSession._do_run\u001B[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001B[0m\n\u001B[1;32m   1367\u001B[0m   \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_call_tf_sessionprun(handle, feed_dict, fetch_list)\n\u001B[1;32m   1369\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m handle \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m-> 1370\u001B[0m   \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_do_call\u001B[49m\u001B[43m(\u001B[49m\u001B[43m_run_fn\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfeeds\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfetches\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtargets\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43moptions\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1371\u001B[0m \u001B[43m                       \u001B[49m\u001B[43mrun_metadata\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1372\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m   1373\u001B[0m   \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_do_call(_prun_fn, handle, feeds, fetches)\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:1377\u001B[0m, in \u001B[0;36mBaseSession._do_call\u001B[0;34m(self, fn, *args)\u001B[0m\n\u001B[1;32m   1375\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_do_call\u001B[39m(\u001B[38;5;28mself\u001B[39m, fn, \u001B[38;5;241m*\u001B[39margs):\n\u001B[1;32m   1376\u001B[0m   \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1377\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mfn\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1378\u001B[0m   \u001B[38;5;28;01mexcept\u001B[39;00m errors\u001B[38;5;241m.\u001B[39mOpError \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m   1379\u001B[0m     message \u001B[38;5;241m=\u001B[39m compat\u001B[38;5;241m.\u001B[39mas_text(e\u001B[38;5;241m.\u001B[39mmessage)\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:1360\u001B[0m, in \u001B[0;36mBaseSession._do_run.<locals>._run_fn\u001B[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001B[0m\n\u001B[1;32m   1357\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_run_fn\u001B[39m(feed_dict, fetch_list, target_list, options, run_metadata):\n\u001B[1;32m   1358\u001B[0m   \u001B[38;5;66;03m# Ensure any changes to the graph are reflected in the runtime.\u001B[39;00m\n\u001B[1;32m   1359\u001B[0m   \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_extend_graph()\n\u001B[0;32m-> 1360\u001B[0m   \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_call_tf_sessionrun\u001B[49m\u001B[43m(\u001B[49m\u001B[43moptions\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfeed_dict\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfetch_list\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1361\u001B[0m \u001B[43m                                  \u001B[49m\u001B[43mtarget_list\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrun_metadata\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:1453\u001B[0m, in \u001B[0;36mBaseSession._call_tf_sessionrun\u001B[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001B[0m\n\u001B[1;32m   1451\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_call_tf_sessionrun\u001B[39m(\u001B[38;5;28mself\u001B[39m, options, feed_dict, fetch_list, target_list,\n\u001B[1;32m   1452\u001B[0m                         run_metadata):\n\u001B[0;32m-> 1453\u001B[0m   \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mtf_session\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mTF_SessionRun_wrapper\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_session\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43moptions\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfeed_dict\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1454\u001B[0m \u001B[43m                                          \u001B[49m\u001B[43mfetch_list\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtarget_list\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1455\u001B[0m \u001B[43m                                          \u001B[49m\u001B[43mrun_metadata\u001B[49m\u001B[43m)\u001B[49m\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "with tf.Session(graph=g) as sess:\n",
    "    train(sess,\n",
    "          training_set=(X_train, y_train),\n",
    "          validation_set=(X_valid, y_valid),\n",
    "          initialize=True,\n",
    "          random_seed=123)\n",
    "    save(saver, sess, epoch=20)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "g2 = tf.Graph()\n",
    "with g2.as_default():\n",
    "    tf.set_random_seed(random_seed)\n",
    "    ## build the graph\n",
    "    build_cnn()\n",
    "\n",
    "    ## saver:\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "## create a new session\n",
    "## and restore the model\n",
    "with tf.Session(graph=g2) as sess:\n",
    "    load(saver, sess,\n",
    "         epoch=20, path='./model/')\n",
    "\n",
    "    preds = predict(sess, X_test_centered,\n",
    "                    return_proba=False)\n",
    "\n",
    "    print('Test Accuracy: %.3f%%' % (100*\n",
    "                np.sum(preds == y_test)/len(y_test)))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "np.set_printoptions(precision=2, suppress=True)\n",
    "\n",
    "with tf.Session(graph=g2) as sess:\n",
    "    load(saver, sess,\n",
    "         epoch=20, path='./model/')\n",
    "\n",
    "    print(predict(sess, X_test_centered[:10],\n",
    "              return_proba=False))\n",
    "\n",
    "    print(predict(sess, X_test_centered[:10],\n",
    "                  return_proba=True))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "with tf.Session(graph=g2) as sess:\n",
    "    load(saver, sess,\n",
    "         epoch=20, path='./model/')\n",
    "\n",
    "    train(sess,\n",
    "          training_set=(X_train_centered, y_train),\n",
    "          validation_set=(X_valid_centered, y_valid),\n",
    "          initialize=False,\n",
    "          epochs=20,\n",
    "          random_seed=123)\n",
    "\n",
    "    save(saver, sess, epoch=40, path='./model/')\n",
    "\n",
    "    preds = predict(sess, X_test_centered,\n",
    "                    return_proba=False)\n",
    "\n",
    "    print('Test Accuracy: %.3f%%' % (100*\n",
    "                np.sum(preds == y_test)/len(y_test)))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 使用tensorflow的layers的API来实现CNN"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "import os\n",
    "class ConvNN(object):\n",
    "    def __init__(self, batchsize=64,\n",
    "                 epochs=20, learning_rate=1e-4,\n",
    "                 dropout_rate=0.5,\n",
    "                 shuffle=True, random_seed=None):\n",
    "        np.random.seed(random_seed)\n",
    "        self.batchsize = batchsize#批量样本数量\n",
    "        self.epochs = epochs#迭代次数\n",
    "        self.learning_rate = learning_rate#学习率\n",
    "        self.dropout_rate = dropout_rate#删除率\n",
    "        self.shuffle = shuffle#重排序\n",
    "\n",
    "        g = tf.Graph()\n",
    "        with g.as_default():\n",
    "            ## set random-seed:\n",
    "            tf.set_random_seed(random_seed)\n",
    "\n",
    "            ## build the network:\n",
    "            self.build()\n",
    "\n",
    "            ## initializer\n",
    "            self.init_op =                 tf.global_variables_initializer()\n",
    "\n",
    "            ## saver\n",
    "            self.saver = tf.train.Saver()\n",
    "\n",
    "        ## create a session\n",
    "        self.sess = tf.Session(graph=g)\n",
    "\n",
    "    def build(self):\n",
    "\n",
    "        ## Placeholders for X and y:\n",
    "        tf_x = tf.placeholder(tf.float32,\n",
    "                              shape=[None, 784],\n",
    "                              name='tf_x')\n",
    "        tf_y = tf.placeholder(tf.int32,\n",
    "                              shape=[None],\n",
    "                              name='tf_y')\n",
    "        is_train = tf.placeholder(tf.bool,\n",
    "                              shape=(),\n",
    "                              name='is_train')\n",
    "\n",
    "        ## reshape x to a 4D tensor:\n",
    "        ##  [batchsize, width, height, 1]\n",
    "        tf_x_image = tf.reshape(tf_x, shape=[-1, 28, 28, 1],\n",
    "                              name='input_x_2dimages')\n",
    "        ## One-hot encoding:\n",
    "        tf_y_onehot = tf.one_hot(indices=tf_y, depth=10,\n",
    "                              dtype=tf.float32,\n",
    "                              name='input_y_onehot')\n",
    "\n",
    "        ## 1st layer: Conv_1\n",
    "        h1 = tf.layers.conv2d(tf_x_image,\n",
    "                              kernel_size=(5, 5),\n",
    "                              filters=32,\n",
    "                              activation=tf.nn.relu)\n",
    "        ## MaxPooling\n",
    "        h1_pool = tf.layers.max_pooling2d(h1,\n",
    "                              pool_size=(2, 2),\n",
    "                              strides=(2, 2))\n",
    "        ## 2n layer: Conv_2\n",
    "        h2 = tf.layers.conv2d(h1_pool, kernel_size=(5,5),\n",
    "                              filters=64,\n",
    "                              activation=tf.nn.relu)\n",
    "        ## MaxPooling\n",
    "        h2_pool = tf.layers.max_pooling2d(h2,\n",
    "                              pool_size=(2, 2),\n",
    "                              strides=(2, 2))\n",
    "\n",
    "        ## 3rd layer: Fully Connected\n",
    "        input_shape = h2_pool.get_shape().as_list()\n",
    "        n_input_units = np.prod(input_shape[1:])\n",
    "        h2_pool_flat = tf.reshape(h2_pool,\n",
    "                              shape=[-1, n_input_units])\n",
    "        h3 = tf.layers.dense(h2_pool_flat, 1024,\n",
    "                              activation=tf.nn.relu)\n",
    "\n",
    "        ## Dropout\n",
    "        h3_drop = tf.layers.dropout(h3,\n",
    "                              rate=self.dropout_rate,\n",
    "                              training=is_train)\n",
    "\n",
    "        ## 4th layer: Fully Connected (linear activation)\n",
    "        h4 = tf.layers.dense(h3_drop, 10,\n",
    "                              activation=None)\n",
    "\n",
    "        ## Prediction\n",
    "        predictions = {\n",
    "            'probabilities': tf.nn.softmax(h4,\n",
    "                              name='probabilities'),\n",
    "            'labels': tf.cast(tf.argmax(h4, axis=1),\n",
    "                              tf.int32, name='labels')}\n",
    "\n",
    "        ## Loss Function and Optimization\n",
    "        cross_entropy_loss = tf.reduce_mean(\n",
    "            tf.nn.softmax_cross_entropy_with_logits(\n",
    "                logits=h4, labels=tf_y_onehot),\n",
    "            name='cross_entropy_loss')\n",
    "\n",
    "        ## Optimizer\n",
    "        optimizer = tf.train.AdamOptimizer(self.learning_rate)\n",
    "        optimizer = optimizer.minimize(cross_entropy_loss,\n",
    "                                       name='train_op')\n",
    "\n",
    "        ## Finding accuracy\n",
    "        correct_predictions = tf.equal(\n",
    "            predictions['labels'],\n",
    "            tf_y, name='correct_preds')\n",
    "\n",
    "        accuracy = tf.reduce_mean(\n",
    "            tf.cast(correct_predictions, tf.float32),\n",
    "            name='accuracy')\n",
    "\n",
    "    def save(self, epoch, path='./tflayers-model/'):\n",
    "        if not os.path.isdir(path):\n",
    "            os.makedirs(path)\n",
    "        print('Saving model in %s' % path)\n",
    "        self.saver.save(self.sess,\n",
    "                        os.path.join(path, 'model.ckpt'),\n",
    "                        global_step=epoch)\n",
    "\n",
    "    def load(self, epoch, path):\n",
    "        print('Loading model from %s' % path)\n",
    "        self.saver.restore(self.sess,\n",
    "             os.path.join(path, 'model.ckpt-%d' % epoch))\n",
    "\n",
    "    def train(self, training_set,\n",
    "              validation_set=None,\n",
    "              initialize=True):\n",
    "        ## initialize variables\n",
    "        if initialize:\n",
    "            self.sess.run(self.init_op)\n",
    "\n",
    "        self.train_cost_ = []\n",
    "        X_data = np.array(training_set[0])\n",
    "        y_data = np.array(training_set[1])\n",
    "\n",
    "        for epoch in range(1, self.epochs + 1):\n",
    "            batch_gen =                 batch_generator(X_data, y_data,\n",
    "                                 shuffle=self.shuffle)\n",
    "            avg_loss = 0.0\n",
    "            for i, (batch_x,batch_y) in                 enumerate(batch_gen):\n",
    "                feed = {'tf_x:0': batch_x,\n",
    "                        'tf_y:0': batch_y,\n",
    "                        'is_train:0': True} ## for dropout\n",
    "                loss, _ = self.sess.run(\n",
    "                        ['cross_entropy_loss:0', 'train_op'],\n",
    "                        feed_dict=feed)\n",
    "                avg_loss += loss\n",
    "\n",
    "            print('Epoch %02d: Training Avg. Loss: '\n",
    "                  '%7.3f' % (epoch, avg_loss), end=' ')\n",
    "            if validation_set is not None:\n",
    "                feed = {'tf_x:0': batch_x,\n",
    "                        'tf_y:0': batch_y,\n",
    "                        'is_train:0': False} ## for dropout\n",
    "                valid_acc = self.sess.run('accuracy:0',\n",
    "                                          feed_dict=feed)\n",
    "                print('Validation Acc: %7.3f' % valid_acc)\n",
    "            else:\n",
    "                print()\n",
    "\n",
    "    def predict(self, X_test, return_proba = False):\n",
    "        feed = {'tf_x:0': X_test,\n",
    "                'is_train:0': False} ## for dropout\n",
    "        if return_proba:\n",
    "            return self.sess.run('probabilities:0',\n",
    "                                 feed_dict=feed)\n",
    "        else:\n",
    "            return self.sess.run('labels:0',\n",
    "                                 feed_dict=feed)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/b6/sj0f93ps3rn4fdfsd4mkgj_00000gn/T/ipykernel_71924/2537897753.py:54: UserWarning: `tf.layers.conv2d` is deprecated and will be removed in a future version. Please Use `tf.keras.layers.Conv2D` instead.\n",
      "  h1 = tf.layers.conv2d(tf_x_image,\n",
      "/Users/apple/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/keras/legacy_tf_layers/convolutional.py:575: UserWarning: `layer.apply` is deprecated and will be removed in a future version. Please use `layer.__call__` method instead.\n",
      "  return layer.apply(inputs)\n",
      "/var/folders/b6/sj0f93ps3rn4fdfsd4mkgj_00000gn/T/ipykernel_71924/2537897753.py:59: UserWarning: `tf.layers.max_pooling2d` is deprecated and will be removed in a future version. Please use `tf.keras.layers.MaxPooling2D` instead.\n",
      "  h1_pool = tf.layers.max_pooling2d(h1,\n",
      "/Users/apple/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/keras/legacy_tf_layers/pooling.py:600: UserWarning: `layer.apply` is deprecated and will be removed in a future version. Please use `layer.__call__` method instead.\n",
      "  return layer.apply(inputs)\n",
      "/var/folders/b6/sj0f93ps3rn4fdfsd4mkgj_00000gn/T/ipykernel_71924/2537897753.py:63: UserWarning: `tf.layers.conv2d` is deprecated and will be removed in a future version. Please Use `tf.keras.layers.Conv2D` instead.\n",
      "  h2 = tf.layers.conv2d(h1_pool, kernel_size=(5,5),\n",
      "/var/folders/b6/sj0f93ps3rn4fdfsd4mkgj_00000gn/T/ipykernel_71924/2537897753.py:67: UserWarning: `tf.layers.max_pooling2d` is deprecated and will be removed in a future version. Please use `tf.keras.layers.MaxPooling2D` instead.\n",
      "  h2_pool = tf.layers.max_pooling2d(h2,\n",
      "/var/folders/b6/sj0f93ps3rn4fdfsd4mkgj_00000gn/T/ipykernel_71924/2537897753.py:76: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  h3 = tf.layers.dense(h2_pool_flat, 1024,\n",
      "/Users/apple/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/keras/legacy_tf_layers/core.py:261: UserWarning: `layer.apply` is deprecated and will be removed in a future version. Please use `layer.__call__` method instead.\n",
      "  return layer.apply(inputs)\n",
      "/var/folders/b6/sj0f93ps3rn4fdfsd4mkgj_00000gn/T/ipykernel_71924/2537897753.py:80: UserWarning: `tf.layers.dropout` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dropout` instead.\n",
      "  h3_drop = tf.layers.dropout(h3,\n",
      "/Users/apple/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/keras/legacy_tf_layers/core.py:413: UserWarning: `layer.apply` is deprecated and will be removed in a future version. Please use `layer.__call__` method instead.\n",
      "  return layer.apply(inputs, training=training)\n",
      "/var/folders/b6/sj0f93ps3rn4fdfsd4mkgj_00000gn/T/ipykernel_71924/2537897753.py:85: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  h4 = tf.layers.dense(h3_drop, 10,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/apple/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/util/dispatch.py:1082: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-05-23 19:56:46.585381: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "cnn=ConvNN(random_seed=123)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 01: Training Avg. Loss: 308.492 Validation Acc:   0.938\n",
      "Epoch 02: Training Avg. Loss:  80.949 Validation Acc:   1.000\n",
      "Epoch 03: Training Avg. Loss:  53.947 Validation Acc:   1.000\n",
      "Epoch 04: Training Avg. Loss:  42.224 Validation Acc:   1.000\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Input \u001B[0;32mIn [9]\u001B[0m, in \u001B[0;36m<cell line: 1>\u001B[0;34m()\u001B[0m\n\u001B[0;32m----> 1\u001B[0m \u001B[43mcnn\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtrain\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtraining_set\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mX_train\u001B[49m\u001B[43m,\u001B[49m\u001B[43my_train\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m      2\u001B[0m \u001B[43m          \u001B[49m\u001B[43mvalidation_set\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mX_valid\u001B[49m\u001B[43m,\u001B[49m\u001B[43my_valid\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m      3\u001B[0m \u001B[43m          \u001B[49m\u001B[43minitialize\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[1;32m      4\u001B[0m cnn\u001B[38;5;241m.\u001B[39msave(epoch\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m20\u001B[39m)\n",
      "Input \u001B[0;32mIn [2]\u001B[0m, in \u001B[0;36mConvNN.train\u001B[0;34m(self, training_set, validation_set, initialize)\u001B[0m\n\u001B[1;32m    143\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m i, (batch_x,batch_y) \u001B[38;5;129;01min\u001B[39;00m                 \u001B[38;5;28menumerate\u001B[39m(batch_gen):\n\u001B[1;32m    144\u001B[0m     feed \u001B[38;5;241m=\u001B[39m {\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mtf_x:0\u001B[39m\u001B[38;5;124m'\u001B[39m: batch_x,\n\u001B[1;32m    145\u001B[0m             \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mtf_y:0\u001B[39m\u001B[38;5;124m'\u001B[39m: batch_y,\n\u001B[1;32m    146\u001B[0m             \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mis_train:0\u001B[39m\u001B[38;5;124m'\u001B[39m: \u001B[38;5;28;01mTrue\u001B[39;00m} \u001B[38;5;66;03m## for dropout\u001B[39;00m\n\u001B[0;32m--> 147\u001B[0m     loss, _ \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msess\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrun\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m    148\u001B[0m \u001B[43m            \u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mcross_entropy_loss:0\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mtrain_op\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    149\u001B[0m \u001B[43m            \u001B[49m\u001B[43mfeed_dict\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mfeed\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    150\u001B[0m     avg_loss \u001B[38;5;241m+\u001B[39m\u001B[38;5;241m=\u001B[39m loss\n\u001B[1;32m    152\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mEpoch \u001B[39m\u001B[38;5;132;01m%02d\u001B[39;00m\u001B[38;5;124m: Training Avg. Loss: \u001B[39m\u001B[38;5;124m'\u001B[39m\n\u001B[1;32m    153\u001B[0m       \u001B[38;5;124m'\u001B[39m\u001B[38;5;132;01m%7.3f\u001B[39;00m\u001B[38;5;124m'\u001B[39m \u001B[38;5;241m%\u001B[39m (epoch, avg_loss), end\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m \u001B[39m\u001B[38;5;124m'\u001B[39m)\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:967\u001B[0m, in \u001B[0;36mBaseSession.run\u001B[0;34m(self, fetches, feed_dict, options, run_metadata)\u001B[0m\n\u001B[1;32m    964\u001B[0m run_metadata_ptr \u001B[38;5;241m=\u001B[39m tf_session\u001B[38;5;241m.\u001B[39mTF_NewBuffer() \u001B[38;5;28;01mif\u001B[39;00m run_metadata \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[1;32m    966\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 967\u001B[0m   result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_run\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfetches\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfeed_dict\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43moptions_ptr\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    968\u001B[0m \u001B[43m                     \u001B[49m\u001B[43mrun_metadata_ptr\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    969\u001B[0m   \u001B[38;5;28;01mif\u001B[39;00m run_metadata:\n\u001B[1;32m    970\u001B[0m     proto_data \u001B[38;5;241m=\u001B[39m tf_session\u001B[38;5;241m.\u001B[39mTF_GetBuffer(run_metadata_ptr)\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:1190\u001B[0m, in \u001B[0;36mBaseSession._run\u001B[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001B[0m\n\u001B[1;32m   1187\u001B[0m \u001B[38;5;66;03m# We only want to really perform the run if fetches or targets are provided,\u001B[39;00m\n\u001B[1;32m   1188\u001B[0m \u001B[38;5;66;03m# or if the call is a partial run that specifies feeds.\u001B[39;00m\n\u001B[1;32m   1189\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m final_fetches \u001B[38;5;129;01mor\u001B[39;00m final_targets \u001B[38;5;129;01mor\u001B[39;00m (handle \u001B[38;5;129;01mand\u001B[39;00m feed_dict_tensor):\n\u001B[0;32m-> 1190\u001B[0m   results \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_do_run\u001B[49m\u001B[43m(\u001B[49m\u001B[43mhandle\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfinal_targets\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfinal_fetches\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1191\u001B[0m \u001B[43m                         \u001B[49m\u001B[43mfeed_dict_tensor\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43moptions\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrun_metadata\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1192\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m   1193\u001B[0m   results \u001B[38;5;241m=\u001B[39m []\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:1370\u001B[0m, in \u001B[0;36mBaseSession._do_run\u001B[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001B[0m\n\u001B[1;32m   1367\u001B[0m   \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_call_tf_sessionprun(handle, feed_dict, fetch_list)\n\u001B[1;32m   1369\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m handle \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m-> 1370\u001B[0m   \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_do_call\u001B[49m\u001B[43m(\u001B[49m\u001B[43m_run_fn\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfeeds\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfetches\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtargets\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43moptions\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1371\u001B[0m \u001B[43m                       \u001B[49m\u001B[43mrun_metadata\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1372\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m   1373\u001B[0m   \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_do_call(_prun_fn, handle, feeds, fetches)\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:1377\u001B[0m, in \u001B[0;36mBaseSession._do_call\u001B[0;34m(self, fn, *args)\u001B[0m\n\u001B[1;32m   1375\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_do_call\u001B[39m(\u001B[38;5;28mself\u001B[39m, fn, \u001B[38;5;241m*\u001B[39margs):\n\u001B[1;32m   1376\u001B[0m   \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1377\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mfn\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1378\u001B[0m   \u001B[38;5;28;01mexcept\u001B[39;00m errors\u001B[38;5;241m.\u001B[39mOpError \u001B[38;5;28;01mas\u001B[39;00m e:\n\u001B[1;32m   1379\u001B[0m     message \u001B[38;5;241m=\u001B[39m compat\u001B[38;5;241m.\u001B[39mas_text(e\u001B[38;5;241m.\u001B[39mmessage)\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:1360\u001B[0m, in \u001B[0;36mBaseSession._do_run.<locals>._run_fn\u001B[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001B[0m\n\u001B[1;32m   1357\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_run_fn\u001B[39m(feed_dict, fetch_list, target_list, options, run_metadata):\n\u001B[1;32m   1358\u001B[0m   \u001B[38;5;66;03m# Ensure any changes to the graph are reflected in the runtime.\u001B[39;00m\n\u001B[1;32m   1359\u001B[0m   \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_extend_graph()\n\u001B[0;32m-> 1360\u001B[0m   \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_call_tf_sessionrun\u001B[49m\u001B[43m(\u001B[49m\u001B[43moptions\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfeed_dict\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfetch_list\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1361\u001B[0m \u001B[43m                                  \u001B[49m\u001B[43mtarget_list\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrun_metadata\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/opt/anaconda3/envs/Deep-Learning/lib/python3.10/site-packages/tensorflow/python/client/session.py:1453\u001B[0m, in \u001B[0;36mBaseSession._call_tf_sessionrun\u001B[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001B[0m\n\u001B[1;32m   1451\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_call_tf_sessionrun\u001B[39m(\u001B[38;5;28mself\u001B[39m, options, feed_dict, fetch_list, target_list,\n\u001B[1;32m   1452\u001B[0m                         run_metadata):\n\u001B[0;32m-> 1453\u001B[0m   \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mtf_session\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mTF_SessionRun_wrapper\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_session\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43moptions\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mfeed_dict\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1454\u001B[0m \u001B[43m                                          \u001B[49m\u001B[43mfetch_list\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtarget_list\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m   1455\u001B[0m \u001B[43m                                          \u001B[49m\u001B[43mrun_metadata\u001B[49m\u001B[43m)\u001B[49m\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "cnn.train(training_set=(X_train,y_train),\n",
    "          validation_set=(X_valid,y_valid),\n",
    "          initialize=True)\n",
    "cnn.save(epoch=20)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "del cnn\n",
    "cnn2=ConvNN(random_seed=123)\n",
    "cnn2.load(epoch=20,path='./tflayers-model/')\n",
    "print(cnn2.predict(X_test[:10,:]))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#计算准确率\n",
    "preds=cnn2.predict(X_test)\n",
    "print('Test Accuracy: %.2f%%' % (100*\n",
    "      np.sum(y_test == preds)/len(y_test)))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 本章学习了CNN或卷积神经网络，并探索了构建不同CNN体系结构所需要的模块。从定义卷积运算开始，然后通过讨论一维和二维的实现了解了它的基本原理。通过讨论最大池和平均池两种形式的池操作，我们也介绍了子采样。然后，将所有这些模块放在一起构建了深度卷积神经网络，用TensorFlow的核心API和Layers API进行了实现，并将其应用于图像识别。"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}